package com.crawler.pageProcessor;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;

import java.util.Collections;
import java.util.List;

public class CnBlogPageProcessor implements PageProcessor {

    // 部分一：抓取网站的相关配置，包括编码、抓取间隔、重试次数等
    private Site site = Site.me().setRetryTimes(3).setSleepTime(1000);


    @Override
    public void process(Page page) {
        List<String> blogTitle = page.getHtml().xpath("//*[@id=\"post_list\"]/article/section/div/a/text()").all();
        List<String> blogSrc = page.getHtml().xpath("//*[@id=\"post_list\"]/article/section/div/a/@href").all();

        // print
        for (int i = 0; i < blogTitle.size(); i++) {
            System.out.println("title:" + blogTitle.get(i));
            System.out.println("src:" + blogSrc.get(i));
            System.out.println();
        }

        // 添加下一页的URL，继续爬取
        List<String> links = page.getHtml().links()
                .regex("https://www.cnblogs.com/sitehome/p/\\d").all();
        if (links.size() > 0) {
            System.out.println("**************************************");
            page.addTargetRequests(Collections.singletonList(links.get(links.size() - 1)));
        }
    }


    @Override
    public Site getSite() {
        return site;
    }
}
